package crawl;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.csvreader.CsvWriter;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;

/**
 * 〈爬取氨基酸序列主类〉
 *
 * @author shiweixian
 * @date 2018/3/10
 * @since 1.0.0
 */
public class Crawler {

    public static void main(String[] args) {
        int number = 0;
        String baseUrl = "http://www.cazy.org/GH11_characterized.html?debut_FUNC=";
        try {
            //--------爬取表格所有链接--------------
            Map<String, String> genBankMap = new HashMap<String, String>(100);
            Map<String, String> uniprotMap = new HashMap<String, String>(100);
            Map<String, String> cookies = Jsoup.connect("http://www.cazy.org/").headers(headers())
                    .ignoreContentType(true).execute().cookies();
            for (int j = 0; j < 3; j++) {
                number = j * 100;
                String url = baseUrl + number + "#pagination_FUNC";
                Connection.Response response = Jsoup.connect(url).headers(headers()).ignoreContentType(true).execute();
                Document document = response.parse();
                Elements elements = document.select("#pos_onglet").first()
                        .select("[valign=top]");
                for (int i = 0; i < elements.size(); i++) {
                    Element tr = elements.get(i);
                    String genBank = tr.children().get(3).html();
                    if (genBank.contains("href=")) {
                        String link = genBank.substring(genBank.indexOf("href=\"") + 6, genBank.indexOf("\" target"));
                        String name = genBank.substring(genBank.indexOf("<b>") + 3, genBank.indexOf("</b>"));
                        genBankMap.put(name, link);
                    }
                    String uniprot = tr.children().get(4).html();
                    if (uniprot.contains("href=")) {
                        String link = uniprot.substring(uniprot.indexOf("href=\"") + 6, uniprot.indexOf("\" target"));
                        String name = link.substring(link.lastIndexOf("/") + 1);
                        uniprotMap.put(name, link);
                    }
                }
                Thread.sleep(500);
            }

            //-------第一层爬取结束-----------------
            //-------爬取氨基酸序列开始-------------
            CsvWriter genWriter = new CsvWriter("d:\\genBank2.csv", ',', Charset.forName("UTF-8"));
            CsvWriter uniWriter = new CsvWriter("d:\\uniport2.csv", ',', Charset.forName("UTF-8"));
            String[] csvHeaders = {"name", "sequences"};
            genWriter.writeRecord(csvHeaders);
            uniWriter.writeRecord(csvHeaders);
            //爬取genBank数据
            String genTail = "?report=fasta";
            String genPre = "<meta name=\"ncbi_uidlist\" content=\"";
            String genAfter = "\">\n" +
                    "  <meta name=\"ncbi_filter\"";
            int count = 1;
            for (String name : genBankMap.keySet()) {
                println("第" + count + "个");
                String url = genBankMap.get(name);
                println(url);
                Document genDoc = Jsoup.connect(url).ignoreContentType(true).cookies(cookies).get();
                String genH = genDoc.html();
                String id = genH.substring(genH.indexOf(genPre) + genPre.length(), genH.indexOf(genAfter));
                println(id);
                String hUrl = "https://www.ncbi.nlm.nih.gov/sviewer/viewer.fcgi?db=protein&report=fasta" +
                        "&extrafeat=0&conwithfeat=on&retmode=html&withmarkup=on&tool=portal&log$=seqview&maxdownloadsize=1000000";
                String data = Jsoup.connect(hUrl).ignoreContentType(true).data("id", id).execute().body().trim();
                String[] datas = data.split("\n");
                StringBuilder acid = new StringBuilder();
                for (int i = 1; i < datas.length; i++) {
                    acid.append(datas[i]);
                }
                println(name);
                println(acid.toString());
                genWriter.writeRecord(new String[]{name, acid.toString()});
                genWriter.flush();
                count++;
                Thread.sleep(1000);
            }

            println("第一部分写出完成");
            count = 1;

            //爬取uniport数据
            String uniTail = ".fasta";
            for (String name : uniprotMap.keySet()) {
                println("第" + count + "个");
                String data = Jsoup.connect(uniprotMap.get(name) + uniTail).ignoreContentType(true).cookies(cookies)
                        .execute().body().trim();
                String[] datas = data.split("\n");
                StringBuilder acid = new StringBuilder();
                for (int i = 1; i < datas.length; i++) {
                    acid.append(datas[i]);
                }
                println(name);
                println(acid.toString());
                uniWriter.writeRecord(new String[]{name, acid.toString()});
                uniWriter.flush();
                count++;
                Thread.sleep(1000);
            }
            genWriter.close();
            uniWriter.close();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    private static Map<String, String> headers() {
        Map<String, String> headers = new HashMap<String, String>();
        headers.put("Acccept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        headers.put("Cache-Control", "max-age=0");
        headers.put("Host", "www.cazy.org");
        headers.put("Proxy-Connection", "keep-alive");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36");
        return headers;
    }

    private static void print(String s) {
        System.out.print(s + "  ");
    }

    private static void print(int s) {
        System.out.print(s + "  ");
    }

    private static void println(String s) {
        System.out.println(s);
    }

    private static void println(int s) {
        System.out.println(s);
    }

}